import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from python_ai.common.xcommon import sep
pd.set_option('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', 1000, 'display.expand_frame_repr', False)
plt.rcParams['font.sans-serif'] = ['Simhei']
plt.rcParams['axes.unicode_minus'] = False

spr = 2
spc = 2
spn = 0
plt.figure(figsize=[12, 12])

sep()
df = pd.read_csv(r'../../../../../large_data/ML2/train_titanic.csv')
m, n = df.shape
np.random.seed(666)
idx_rnd = np.random.permutation(m)
df = df.iloc[idx_rnd]
print(df.shape)
print(df[:5])

sep('Check na')
print(type(df.isnull()))
print(df.isnull()[:5])
print(df.isnull().sum(axis=0))
print(df.columns[df.isna().sum(axis=0) > 0])

sep('Check and remove PassengerId')
print("type(df['PassengerId'])", type(df['PassengerId']))
print("type(df['PassengerId'].value_counts())", type(df['PassengerId'].value_counts()))
print("type(df['PassengerId'].value_counts().value_counts())", type(df['PassengerId'].value_counts().value_counts()))
print(df['PassengerId'].value_counts().value_counts())
df.drop(labels='PassengerId', axis=1, inplace=True)
print(df[:5])

sep('Pclass')
analysis_pclass = df['Pclass'].value_counts()
print(analysis_pclass.sort_index())
idx_survived = df['Survived'] == 1
idx_dead = df['Survived'] == 0
analysis_pclass_survived = df.loc[idx_survived, 'Pclass'].value_counts()
analysis_pclass_dead = df.loc[idx_dead, 'Pclass'].value_counts()
spn += 1
ax = plt.subplot(spr, spc, spn)
plt.title('by Pclass')
pd.DataFrame({'生还': analysis_pclass_survived, '遇难': analysis_pclass_dead}).plot(kind='bar', ax=ax)
print('dead')
print(analysis_pclass_dead.sort_index())
print('survived')
print(analysis_pclass_survived.sort_index())
print('survived rate')
print((analysis_pclass_survived/analysis_pclass).sort_index())

sep('Name analysis')
import re


def x_extract_title(name):
    """
    https://docs.python.org/3/library/re.html

    re.sub(pattern, repl, string, count=0, flags=0) Return the string obtained by replacing the leftmost
    non-overlapping occurrences of pattern in string by the replacement repl.

    :param name:
    :return:
    """
    if not hasattr(x_extract_title, 'regexp1'):
        x_extract_title.regexp1 = re.compile(r'^.*, ')
        x_extract_title.regexp2 = re.compile(r'\..*$')
    name = re.sub(x_extract_title.regexp1, '', name)
    name = re.sub(x_extract_title.regexp2, '', name)
    return name


df['title'] = df['Name'].map(x_extract_title)
print(df[:5])
print(df['title'].value_counts().sort_values(ascending=False))

sep('Name analysis (my approach)')


def x_extract_title_my(name):
    """
    https://docs.python.org/3/library/re.html

    re.match(pattern, string, flags=0) If zero or more characters at the beginning of string match the regular
    expression pattern, return a corresponding match object. Return None if the string does not match the pattern;
    note that this is different from a zero-length match.

    Note that even in MULTILINE mode, re.match() will only match at the beginning of the string and not at the beginning
    of each line.

    If you want to locate a match anywhere in string, use search() instead (see also search() vs. match()).

    :param name:
    :return:
    """
    if not hasattr(x_extract_title_my, 'regexp'):
        # x_extract_title_my.regexp = re.compile(r'^[^,]*,\s*([^ .]+)')  # I cannot specify the target does not contain space!
        # Rothes, the Countess. of (Lucy Noel Martha Dyer-Edwards)
        x_extract_title_my.regexp = re.compile(r'^[^,]*,\s*([^.]+)')
    mobj = re.match(x_extract_title_my.regexp, name)
    if mobj is None:
        return ""
    else:
        return mobj[1]


df['title_my'] = df['Name'].map(x_extract_title_my)
print(df[:5])
print(np.unique((df['title'] == df['title_my'])))
print(df[df['title'] != df['title_my']])

sep('Name analysis (my approach 2)')


def x_extract_title_my02(name):
    """
    https://docs.python.org/3/library/re.html

    re.search(pattern, string, flags=0) Scan through string looking for the first location where the regular
    expression pattern produces a match, and return a corresponding match object. Return None if no position in the
    string matches the pattern; note that this is different from finding a zero-length match at some point in the
    string.

    :param name:
    :return:
    """
    if not hasattr(x_extract_title_my, 'regexp'):
        # x_extract_title_my.regexp = re.compile(r'^[^,]*,\s*([^ .]+)')  # I cannot specify the target does not contain space!
        # Rothes, the Countess. of (Lucy Noel Martha Dyer-Edwards)
        x_extract_title_my.regexp = re.compile(r',\s*([^.]+)')
    mobj = re.search(x_extract_title_my.regexp, name)
    if mobj is None:
        return ""
    else:
        return mobj[1]


df['title_my02'] = df['Name'].map(x_extract_title_my)
print(df[:5])
print(np.unique((df['title'] == df['title_my02'])))
print(df[df['title'] != df['title_my02']])

sep('Mlle, Ms,  => Miss; Other => Rare')
df.loc[df['title'] == 'Mlle', 'title'] = 'Miss'
df.loc[df['title'] == 'Ms', 'title'] = 'Miss'
df.loc[~df['title'].isin(['Mr', 'Miss', 'Mrs', 'Master']), 'title'] = 'Rare'
print(df['title'].value_counts())

sep('Analyse sex')
analysis_sex = df['Sex'].value_counts()
analysis_sex_survived = df.loc[idx_survived, 'Sex'].value_counts()
analysis_sex_dead = df.loc[idx_dead, 'Sex'].value_counts()
spn += 1
ax = plt.subplot(spr, spc, spn)
pd.DataFrame(dict(生还=analysis_sex_survived, 遇难=analysis_sex_dead))\
    .sort_index(axis=0, ascending=False).plot(kind='bar', ax=ax)


sep('Analyse title')
analysis_title = df['title'].value_counts()
analysis_title_survived = df.loc[idx_survived, 'title'].value_counts()
analysis_title_dead = df.loc[idx_dead, 'title'].value_counts()
spn += 1
ax = plt.subplot(spr, spc, spn)
plt.title('by title')
pd.DataFrame({'生还': analysis_title_survived, '遇难': analysis_title_dead})\
    .sort_index(axis=0, ascending=False).plot(kind='bar', ax=ax)

sep('fillna for Fare')
print(df.info())
idx_fare_na = df['Fare'].isna()
print(df[idx_fare_na])
df['Fare'].fillna(df['Fare'].mean(), inplace=True)  # ATTENTION fillna, inplace
sep()
print(df.info())
print(df[idx_fare_na])

sep('fillna for Embarked')
print(df['Embarked'].describe())
print(df['Embarked'].value_counts())
idx_embarked_na = df['Embarked'].isna()
print(df[idx_embarked_na])
df['Embarked'].fillna(df['Embarked'].describe().top, inplace=True)
print(df.info())
print(df[idx_embarked_na])

sep('Drop Cabin')
df.drop(labels='Cabin', axis=1, inplace=True)
print(df[:5])

sep('Hypothesis of age')
idx_age_na = df['Age'].isna()
idx_age_notna = np.invert(idx_age_na)
print(type(idx_age_na))
print(idx_age_na[:5])
# onehot and scale
df_labels = df[['Survived', 'Pclass', 'Sex', 'Embarked', 'title']]
df_continuous = df[['Fare']]  # ATTENTION to get a matrix, not vector
from sklearn.preprocessing import OneHotEncoder, StandardScaler
x_continuous = StandardScaler().fit_transform(df_continuous)
x_labels = OneHotEncoder().fit_transform(df_labels).toarray()
print(type(x_continuous))
print(type(x_labels))
x = np.c_[x_continuous, x_labels]
# print(x[:5])
# train and predict
x_train = x[idx_age_notna]
x_test = x[idx_age_na]
y_train = df.loc[idx_age_notna, 'Age']
sep('grid search')
from sklearn.linear_model import Ridge
from sklearn.model_selection import GridSearchCV
estimator = Ridge()
grid = GridSearchCV(estimator,
                    dict(alpha=[0.1, 0.2, 0.5, 1, 2, 5]),
                    cv=5)
grid.fit(x_train, y_train)
print(f'Best score: {grid.best_score_}')
print(f'Best params: {grid.best_params_}')
model = Ridge(**(grid.best_params_))
model.fit(x_train, y_train)
h_test = model.predict(x_test)
print(df[idx_age_na][:5])
df.loc[idx_age_na, 'Age'] = h_test
sep()
print(df.info())
print(df[idx_age_na][:5])

sep('Age sections')
df.loc[df['Age'] <= 12, 'age_section'] = 'child'
df.loc[df['Age'].between(13, 40), 'age_section'] = 'adult'
df.loc[df['Age'] > 40, 'age_section'] = 'elder'
print(df['age_section'].value_counts())
print(df[:5])

sep('Age get_dummies')
df = pd.get_dummies(df, columns=['age_section'])
print(df[:5])

# finally show all drawings
plt.show()
